import time
# Retry decorator
def retry(tries, delay=3, timeout=60, validator=None):
    """Retries a function or method until it returns 0,
    usage: 1. retries calling foo(a, b, c)
            retry(3, delay=1, timeout=2)(foo)(a,b,c)

           2. as function decorator to retry time
               @retry(3, delay=1, timeout=2)
               def foo(a, b, c):
                  ...
               when foo(a,b,c) is called, it will be retried

    @param tries: max retry times
    @param delay: sleep between retries
    @param timeout: max time retry
    @param validator: function to verify if need to retry
    @return the last output value of the function
    """

    if tries < 1:
        raise ValueError("tries must be 1 or greater")

    if delay <= 0:
        raise ValueError("delay must be greater than 0")

    if timeout <= 0:
        raise ValueError("timeout must be greater than 0")

    if validator and not hasattr(validator, '__call__'):
        raise ValueError("validator must be a function")

    def deco_retry(f):
        def f_retry(*args, **kwargs):
            tried = 0
            rv = -1
            starttime = time.time()
            while True:
                if tried >= tries:
                    logger.warn("max retry [%d] reached in executing [%s]" % (tries, f.__name__))
                    break
                if time.time() > starttime + timeout:
                    logger.warn("timeout [%d] reached in executing [%s]" % (timeout, f.__name__))
                    break
                try:
                    rv = f(*args, **kwargs)
                    if validator and validator(rv):
                        return rv
                    elif not isinstance(rv, bool) and rv == 0:  # Done on success
                        return 0
                    else:
                        logger.debug("function [%s] returned [%s]" % (f.__name__, rv))
                except Exception, e:
                    logger.exception("hit [%s] in executing [%s]" % (e, f.__name__))

                tried += 1
                time.sleep(delay)
            return rv

        return f_retry

    return deco_retry